{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4260"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data_path = \"D:/数据/train_gpt2/gpt2_data/webtext2019\"\n",
    "\n",
    "all_data_path_list = glob(pathname=all_data_path + \"/*\")\n",
    "len(all_data_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4260it [00:01, 4119.84it/s]\n"
     ]
    }
   ],
   "source": [
    "def sub_data(x:str) -> None:\n",
    "    save_x_path = x.replace(all_data_path, \"data\")\n",
    "    pd.read_csv(x).pipe(\n",
    "    lambda x: x.assign(**{\n",
    "    'sentence':x['content'].apply(lambda j: j[:64]),\n",
    "    'label':x['content'].apply(lambda j: j[64:])\n",
    "    })\n",
    "    )[['sentence', 'label']].pipe(\n",
    "        lambda x: x.loc[x['sentence'].apply(lambda j: len(j) >0)]\n",
    "    ).pipe(\n",
    "        lambda x: x.loc[x['label'].apply(lambda j: len(j) >0)]\n",
    "    ).to_csv(save_x_path, index=False)\n",
    "\n",
    "\n",
    "for index, value in tqdm(enumerate(all_data_path_list)):\n",
    "    if index < 50:\n",
    "\n",
    "        sub_data(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float32"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "import torch\n",
    "data = torch.randn(1,23)\n",
    "data.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.1367,  0.9180,  0.5337, -0.0313,  0.4136, -1.8184,  1.1611, -1.7305,\n",
       "          0.5547,  0.5327,  0.9419,  0.3533,  2.3750,  0.0440,  0.5781, -0.7393,\n",
       "          0.3301,  0.3420,  1.5010,  0.3188,  1.3301, -2.1445, -0.1731]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.randn(1,23).to(torch.float16).to(torch.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "data_dir = \"data\"\n",
    "os.makedirs(name=data_dir,exist_ok=True)\n",
    "\n",
    "for i in range(20):\n",
    "    data = pd.DataFrame({'sentence':['ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，'] * 100})\n",
    "    data.to_csv(f\"{data_dir}/{i}.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mynet",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
